package com.miyu.test.magic;

import com.alibaba.fastjson.JSONObject;
import lombok.SneakyThrows;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.List;

public class MagicDown {

    private static final String FielName = "XiaoYu";

    public static void main(String[] args) {
        JSONObject data = new JSONObject();
        data.put("level","1");
        data.put("rootUrl","https://www.xiurenb.net/");
        String url = "https://www.xiurenb.net/"+FielName+"/";
        Request request = new Request();
        request.setUrl(url);
        request.setCharset("utf-8");
        request.putExtra("info", data);


        Spider.create(new MagicTuDown()).addRequest(request)
               // .addPipeline(new downLoadFile())
                .thread(50).run();
    }

    static class MagicTuDown implements PageProcessor {
        private  int num1=0;
        private  int num2=0;
        private  int num3=0;
        @SneakyThrows
        @Override
        public void process(Page page) {


            JSONObject  extra = page.getRequest().getExtra("info");
            Request request = page.getRequest();
           if(extra.getString("level").equals("1")){
               num1++;
               //第一层列表
               JSONObject info = page.getRequest().getExtra("info");

               List<Selectable> list = page.getHtml().xpath("//ul[@class='update_area_lists']/li").nodes() ;
               List<Selectable> pageList = page.getHtml().xpath("div[@class='page']/a").nodes();
               if(list.size()>0){
                   for (int i = 0; i < list.size(); i++) {
                       JSONObject temp = new JSONObject();
                       temp.putAll(info);
                       Selectable mk = list.get(i);//一个选中
                       Request req = new Request();
                       String url = mk.xpath("//li/a").links().toString();//图片页面 二级
                       String name1 =mk.xpath("//a/div/span/text()").toString();//一级文件夹名称
                       temp.put("level","2");
                       temp.put("name1",name1);//
                       req.setUrl(url);
                       req.putExtra("info",temp);
                       page.addTargetRequest(req);
                       page.setSkip(true);
                   }
                   if(pageList.size()>0){
                       for (int i = 0; i < pageList.size(); i++) {
                           Selectable mk = pageList.get(i);//一个选中
                           String pageNext = mk.xpath("//a/text()").toString();
                           if("下页".equals(pageNext)){
                               JSONObject temp = new JSONObject();
                               temp.putAll(info);
                               Request req = new Request();
                               String url = mk.xpath("/a").links().toString();//下一页
                               temp.put("level","1");
                               req.setUrl(url);
                               req.putExtra("info",temp);
                               page.addTargetRequest(req);
                               page.setSkip(true);
                               break;
                           }
                       }

                   }

               }
           }else
            if(extra.getString("level").equals("2")){
                num2++;
                JSONObject info = page.getRequest().getExtra("info");
                //第二层列表 //只获取列表分v页链接 不下载数据
                //获取二级文件夹名称
                String name2 = page.getHtml().xpath("//div[@class='item_title']/h1/text()").get();
                List<Selectable> list = page.getHtml().xpath("div[@class='page']/a").nodes();
                if(list.size()>0){
                    for (int i = 0; i < list.size(); i++) {
                        JSONObject temp = new JSONObject();
                        Request req = new Request();
                        Selectable mk = list.get(i);//一个选中
                        String url = mk.xpath("//a").links().toString();//图片页面 二级
                        if(url.length()>0){
                            temp.putAll(info);
                            temp.put("level","3");
                            temp.put("name2",name2);//
                            req.setUrl(url);
                            req.putExtra("info",temp);
                            page.addTargetRequest(req);
                            page.setSkip(true);
                        }
                    }


                }
            }else
            if(extra.getString("level").equals("3")){
                num3++;
                JSONObject info = page.getRequest().getExtra("info");
                //独立图片 供下载
                List<Selectable> list = page.getHtml().xpath("//div[@class='content']/p/img").nodes();
                for (int i = 0; i < list.size(); i++) {
                    Selectable psv = list.get(i);//一个选中
                    String imgUrl = psv.xpath("//img/@src").toString();
                    String rootImgUrl = info.getString("rootUrl")+imgUrl;
                    info.put("rootImgUrl",rootImgUrl);
                    this.downLoadFile(info);

                }

            }
            System.out.println("num1执行次数"+num1);
            System.out.println("num2执行次数"+num2);
            System.out.println("num3执行次数"+num3);

        }

        /**
         * 图片下载
         * @param info
         */
        private void downLoadFile(JSONObject info) throws Exception {
            //路径
            String filePath = "H:\\爬虫数据\\图片\\人人\\"+FielName+"\\";
            String name1 = info.getString("name1");
            String name2 = info.getString("name2");
            String url = info.getString("rootImgUrl");
            String imgName = url.substring(url.lastIndexOf("/")+1,url.lastIndexOf("."))+url.substring(url.lastIndexOf("."));
            String path = filePath+name1+"\\"+name2+"\\";
//先创建文件夹
            File fp = new File(path);
            if (!fp.exists()) {
                fp.mkdirs();// 能创建多级目录
            }
            String FN = path+imgName;
            File file = new File(FN.trim());
            if(!file.exists()) {
                file.createNewFile();
            }else {
                return;
            }

            HttpURLConnection connection = null;
            URL ImgUrl = new URL(url);
            URLConnection con = ImgUrl.openConnection();
            InputStream inStream = con.getInputStream();
            if(inStream.available()==0)
                return;
            ByteArrayOutputStream outStream = new ByteArrayOutputStream();
            byte[] buf = new byte[1024];
            int len = 0;
            while((len = inStream.read(buf)) != -1){
                outStream.write(buf,0,len);
            }


            FileOutputStream op = new FileOutputStream(file);
            op.write(outStream.toByteArray());
            inStream.close();
            outStream.close();
            op.close();
        }

        @Override
        public Site getSite() {
            return this.site;
        }

        //爬虫配置信息设置 有条件动态配置
        private final Site site = new Site()
                .setCharset("utf-8")    //设置编码
                .setSleepTime(10)        //设置抓取间隔
                .setTimeOut(100000)         //设置超时时间
                .setRetrySleepTime(3000)      //设置重试时间
                .setRetryTimes(3);       //设置重试次数


    }


}
